PATH = untar_data(URLs.IMAGEWOOF_320)
PATH.ls()

[PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/val'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train')]

data = ImageDataBunch.from_folder(PATH, train='train', valid='val',
                                 ds_tfms=get_transforms(), size=224)

data.show_batch()

len(data.train_ds)

12454

len(data.valid_ds)

500

TRAIN = PATH/'train'

TRAIN.ls()

[PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02093754'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02088364'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02089973'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02087394'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02111889'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02099601'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02105641'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02086240'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02115641'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02096294')]

Make CSV¶

def get_label_from_names(names:Path):
    labels = []
    if is_listy(names):
        for name in names: 
            labels.append(Path(name).parent.name)
        return labels
    else: 
        return Path(names).parent.name
    
names = get_files(TRAIN, recurse=True, extensions='.JPEG')
get_label_from_names(names[:10])

['n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754']

def make_df(names:list):
    names2 = names.copy()
    random.shuffle(names2)
    df = pd.DataFrame(data=[names, names2]).T
    labels1 = np.array(get_label_from_names(list(df.iloc[:,0])))
    labels2 = np.array(get_label_from_names(list(df.iloc[:,1])))
    labels = (labels1 == labels2).astype(np.int32)
    df['label'] = labels
    df.columns = ['image1', 'image2', 'label']
    return df

df = make_df(get_files(TRAIN, recurse=True, extensions='.JPEG'))
df.head()

def make_csv_from_path(TRAIN=TRAIN, mult=5):
    TRAIN = Path(TRAIN)
    names60 = []
    dfs = []
    for d in TRAIN.ls():
        names = d.ls() * mult
        random.shuffle(names)
        names40 = names[:int(len(names)*0.4)]
        names60.extend(names[int(len(names)*0.4):])
        dfs.append(make_df(names40))
        
    df60 = make_df(names60)
    for df in dfs:
        df60 = df60.append(df)

    return df60

df = make_csv_from_path(TRAIN)
df.hist()

array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f14c7d4b790>]], dtype=object)

FastAI Databunch¶

df.head()

class DoubleImageList(ImageList):
    def open(self, fn):
        im1 = open_image(fn[0]).resize(320)
        im2 = open_image(fn[1]).resize(320)
        black_space = torch.zeros(3, 320, 20)
        return Image(torch.cat([im1.data, black_space, im2.data], dim = 2))

data = (DoubleImageList.from_df(df, path='/', cols=['image1', 'image2'])
        .split_by_rand_pct(0.2)
        .label_from_df(cols=['label'])
        .transform(get_transforms())
        .databunch(bs=64)
        .normalize(imagenet_stats))

data.show_batch()

Model¶

predictions = None
preds = None
learn = None
gc.collect()

20

learn = cnn_learner(data, models.resnet18, metrics=[accuracy])

learn.fit_one_cycle(3)

learn.path = Path.cwd()/'models'

learn.load('first')

Learner(data=ImageDataBunch;

Train: LabelList (49816 items)
x: DoubleImageList
Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660)
y: CategoryList
0,0,0,0,0
Path: /;

Valid: LabelList (12454 items)
x: DoubleImageList
Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660)
y: CategoryList
1,1,0,0,1
Path: /;

Test: None, model=Sequential(
  (0): Sequential(
    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU(inplace=True)
    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    (4): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (1): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (5): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (6): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (7): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
  )
  (1): Sequential(
    (0): AdaptiveConcatPool2d(
      (ap): AdaptiveAvgPool2d(output_size=1)
      (mp): AdaptiveMaxPool2d(output_size=1)
    )
    (1): Flatten()
    (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (3): Dropout(p=0.25, inplace=False)
    (4): Linear(in_features=1024, out_features=512, bias=True)
    (5): ReLU(inplace=True)
    (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (7): Dropout(p=0.5, inplace=False)
    (8): Linear(in_features=512, out_features=2, bias=True)
  )
), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f14e0b1ff80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/ubuntu/akash/memory_net/models'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[], layer_groups=[Sequential(
  (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
  (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (2): ReLU(inplace=True)
  (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (4): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (6): ReLU(inplace=True)
  (7): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (8): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (9): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (10): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (11): ReLU(inplace=True)
  (12): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (13): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (14): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
  (15): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (16): ReLU(inplace=True)
  (17): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (18): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (19): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
  (20): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (21): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (22): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (23): ReLU(inplace=True)
  (24): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (25): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
), Sequential(
  (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
  (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (2): ReLU(inplace=True)
  (3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (5): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
  (6): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (7): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (8): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (9): ReLU(inplace=True)
  (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (11): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (12): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
  (13): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (14): ReLU(inplace=True)
  (15): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (16): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (17): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
  (18): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (20): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (21): ReLU(inplace=True)
  (22): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (23): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
), Sequential(
  (0): AdaptiveAvgPool2d(output_size=1)
  (1): AdaptiveMaxPool2d(output_size=1)
  (2): Flatten()
  (3): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (4): Dropout(p=0.25, inplace=False)
  (5): Linear(in_features=1024, out_features=512, bias=True)
  (6): ReLU(inplace=True)
  (7): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (8): Dropout(p=0.5, inplace=False)
  (9): Linear(in_features=512, out_features=2, bias=True)
)], add_time=True, silent=False)

learn.save('first')

Testing¶

Take N images from each class as query images
For each image in validation set:
1. Create sample with each support image
2. Stack in one batch
3. Get predictions
4. Get mean probabilities
5. ARGMAX

TRAIN = TRAIN
VAL = PATH/'val'

num_images = 5
classes = 10
query_images = OrderedDict()
random.seed(42)
for d in TRAIN.ls():
    cls = d.name
    images = d.ls()
    random.shuffle(images)
    images = images[:num_images]
    query_images[cls] = images

samples = get_files(VAL, extensions='.JPEG', recurse=True)

def create_sample_from_paths(path1, path2):
    im1 = open_image(path1).resize(320)
    im2 = open_image(path2).resize(320)
    black_space = torch.zeros(3, 320, 20)
    image = torch.cat([im1.data, black_space, im2.data], dim = 2)
    image = learn.data.norm((image, torch.zeros(1)))[0].unsqueeze(0).cuda()
    return image

from tqdm import tqdm_notebook as tqdm

total, accurate, accuracy = 0.0, 0.0, 0.0
predictions = []
pbar = tqdm(samples)
learn.model.eval()
for sample in pbar:
    label = get_label_from_names(sample)
    positive, negative = [], []
    for key, value in query_images.items():
        if key == label:
            for val in value:
                positive.append(create_sample_from_paths(sample, val))
        else:
            for val in value:
                negative.append(create_sample_from_paths(sample, val))
    positive.extend(negative)
    stack = torch.cat(positive, dim=0)
    predictions.append(learn.model(stack)[:,1].reshape(classes, num_images).detach().cpu())
    preds = predictions[-1].mean(1).argmax().item()
    if preds == 0:
        accurate += 1
    total += 1
    accuracy = accurate / total
    pbar.set_description(f"Accuracy : {accuracy}")
print(accurate/total)

0.848

new_preds = torch.stack(predictions)
new_preds.shape

torch.Size([500, 10, 5])

np.save('predictions.npy', new_preds.numpy())

Path.cwd().ls()

[PosixPath('/home/ubuntu/akash/memory_net/LICENSE'),
 PosixPath('/home/ubuntu/akash/memory_net/setup.py'),
 PosixPath('/home/ubuntu/akash/memory_net/settings.ini'),
 PosixPath('/home/ubuntu/akash/memory_net/docs'),
 PosixPath('/home/ubuntu/akash/memory_net/.git'),
 PosixPath('/home/ubuntu/akash/memory_net/README.md'),
 PosixPath('/home/ubuntu/akash/memory_net/.gitattributes'),
 PosixPath('/home/ubuntu/akash/memory_net/predictions.npy'),
 PosixPath('/home/ubuntu/akash/memory_net/00_core.ipynb'),
 PosixPath('/home/ubuntu/akash/memory_net/.github'),
 PosixPath('/home/ubuntu/akash/memory_net/00_core-Copy1.ipynb'),
 PosixPath('/home/ubuntu/akash/memory_net/index.ipynb'),
 PosixPath('/home/ubuntu/akash/memory_net/CONTRIBUTING.md'),
 PosixPath('/home/ubuntu/akash/memory_net/.gitconfig'),
 PosixPath('/home/ubuntu/akash/memory_net/models'),
 PosixPath('/home/ubuntu/akash/memory_net/.gitignore'),
 PosixPath('/home/ubuntu/akash/memory_net/.ipynb_checkpoints')]

	image1	image2
0	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...
1	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...
2	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...
3	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...
4	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...

	image1	image2	label
0	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...	0
1	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...	1
2	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...	0
3	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...	0
4	/home/ubuntu/.fastai/data/imagewoof-320/train/...	/home/ubuntu/.fastai/data/imagewoof-320/train/...	0

epoch	train_loss	valid_loss	accuracy	time
0	0.406716	0.297715	0.877389	07:52
1	0.291941	0.212959	0.918982	07:53
2	0.245086	0.187337	0.931106	07:54